package tools;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;  
import java.util.Arrays;  
import java.util.List;  
import java.util.regex.Pattern;  
import java.util.regex.Matcher;  
  
/** 
 * Tokenizes strings described in English. 
 * 
 * @author Jun Araki
 * @author Xavier Tannier 
 */  
public class EnglishTokenizer implements Normalizer{
  
    /** A regular expression for letters and numbers. */  
    private static final String regexLetterNumber = "[a-zA-Z0-9]";  
  
    /** A regular expression for non-letters and non-numbers. */  
    private static final String regexNotLetterNumber = "[^a-zA-Z0-9]";  
  
    /** A regular expression for separators. */  
    private static final String regexSeparator = "[\\?!()\";/\\|`]";  
  
    /** A regular expression for separators. */  
    private static final String regexClitics =  
        "'|:|-|'S|'D|'M|'LL|'RE|'VE|N'T|'s|'d|'m|'ll|'re|'ve|n't";  
    
	private ArrayList<String> stopWords;
  
    /** Abbreviations. */  
    private static final List<String> abbrList =  
        Arrays.asList("Co.", "Corp.", "vs.", "e.g.", "etc.", "ex.", "cf.",  
            "eg.", "Jan.", "Feb.", "Mar.", "Apr.", "Jun.", "Jul.", "Aug.",  
            "Sept.", "Oct.", "Nov.", "Dec.", "jan.", "feb.", "mar.",  
            "apr.", "jun.", "jul.", "aug.", "sept.", "oct.", "nov.",  
            "dec.", "ed.", "eds.", "repr.", "trans.", "vol.", "vols.",  
            "rev.", "est.", "b.", "m.", "bur.", "d.", "r.", "M.", "Dept.",  
            "MM.", "U.", "Mr.", "Jr.", "Ms.", "Mme.", "Mrs.", "Dr.",  
            "Ph.D.");  
  
	public EnglishTokenizer() {
		this.stopWords = new ArrayList<String>();
	}
	
	public EnglishTokenizer(String stopWordFileName) throws IOException {
		this.stopWords = new ArrayList<String>();
		//lecture du fichier texte	
		InputStream ips=new FileInputStream(stopWordFileName); 
		InputStreamReader ipsr=new InputStreamReader(ips);
		BufferedReader br=new BufferedReader(ipsr);
		String line;
		while ((line=br.readLine())!=null){
			this.stopWords.add(line);
		}
		br.close(); 
	}
  
    /** 
     * Tokenizes a string using the algorithms by Grefenstette (1999) and 
     * Palmer (2000). 
     */  
    public ArrayList<String> tokenize(String str) {
    	ArrayList<String> tokenList = new ArrayList<String>();

        // Changes tabs into spaces.  
        str = str.replaceAll("\\t", " ");  
  
        // Puts blanks around unambiguous separators.  
        str = str.replaceAll("(" + regexSeparator + ")", " $1 ");  
  
        // Puts blanks around commas that are not inside numbers.  
        str = str.replaceAll("([^0-9]),", "$1 , ");  
        str = str.replaceAll(",([^0-9])", " , $1");  
  
        // Distinguishes single quotes from apstrophes by segmenting off  
        // single quotes not preceded by letters.  
        str = str.replaceAll("^(')", "$1 ");  
        str = str.replaceAll("(" + regexNotLetterNumber + ")'", "$1 '");  
  
        // Segments off unambiguous word-final clitics and punctuations.  
        str = str.replaceAll("(" + regexClitics + ")$", " $1");  
        str = str.replaceAll(  
                "(" + regexClitics + ")(" + regexNotLetterNumber + ")",  
                " $1 $2");  
  
        // Deals with periods.  
        String[] words = str.trim().split("\\s+");  
        Pattern p1 = Pattern.compile(".*" + regexLetterNumber + "\\.");  
        Pattern p2 = Pattern.compile(  
            "^([A-Za-z]\\.([A-Za-z]\\.)+|[A-Z][bcdfghj-nptvxz]+\\.)$");  
        for (String word : words) {  
            Matcher m1 = p1.matcher(word);  
            Matcher m2 = p2.matcher(word);  
            if (m1.matches() && !abbrList.contains(word) && !m2.matches()) {  
                // Segments off the period.  
                tokenList.add(word.substring(0, word.length() - 1));  
                tokenList.add(word.substring(word.length() - 1));  
            } else {  
                tokenList.add(word);  
            }  
        }  
        return tokenList;
    }  
  

	@Override
	public ArrayList<String> normalize(File file) throws IOException {
		return this.normalize(file, false);
	}

	@Override
	public ArrayList<String> normalize(File file, boolean removeStopWords)
			throws IOException {
		String text = "";
		//lecture du fichier texte	
		InputStream ips=new FileInputStream(file); 
		InputStreamReader ipsr=new InputStreamReader(ips);
		BufferedReader br=new BufferedReader(ipsr);
		String line;
		while ((line=br.readLine())!=null){
			text += line + " ";
		}
		br.close(); 
		ArrayList<String> list = this.tokenize(text);
		ArrayList<String> result = new ArrayList<String>();
		for (String word : list) {
			if (!this.stopWords.contains(word)) {
				result.add(word);
			}
		}
		return result;
	}  
	
	@Override
	public ArrayList<String> normalize(String text) throws IOException {
		return this.normalize(text, false);
	}

	@Override
	public ArrayList<String> normalize(String text, boolean removeStopWords)
			throws IOException {
		ArrayList<String> list = this.tokenize(text);
		ArrayList<String> result = new ArrayList<String>();
		for (String word : list) {
			if (!this.stopWords.contains(word)) {
				result.add(word);
			}
		}
		return result;
	}  
}  